package com.splider.toutiao;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.log4j.Logger;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.splider.toutiao.mapper.ArticleMapper;

public class NetEaseTask implements Runnable {

	// 今日头条首页地址
	public final String TOUTIAOURL = "http://www.toutiao.com/api/pc/feed/?category=__all__&utm_source=toutiao&widen=1";
	// 头条号主页地址
	public final String TOUTIAOHAOURL = "http://www.toutiao.com/pgc/ma/?page_type=1&count=10&version=2&platform=pc";

	// 头条号主页地址
	public final String TOUTIAOHAOURL1 = "http://www.toutiao.com/c/user/article/?page_type=1&count=20";

	private SqlSessionFactory sqlSessionFactory = null;

	private SqlSession sqlSession = null;

	private Logger logger = Logger.getLogger("toutiaotask");

	private ArticleMapper articleMapper = null;

	private int totalCount = 0;

	private List<ProxyIP> proxyIps = new ArrayList<ProxyIP>();
	
	private List<ProxyIP> proxyIps_toutiao = new ArrayList<ProxyIP>();

	private List<String> localIps = new ArrayList<String>();

	private String otherCategory = "";

	private Map<String, String> header = new HashMap<String, String>();

	private Map<String, String> header1 = new HashMap<String, String>();

	private String category;
	
	private ProxyIP proxyIP_taojin;
	
	private ProxyIP proxyIP_toutiao;
	
	private int existCount = 0;
	
	
	
	public NetEaseTask(String category,List<ProxyIP> proxyIPs) {
		this.category = category;
		this.proxyIps.addAll(proxyIPs);
		this.proxyIps_toutiao.addAll(proxyIPs);
	}
	
	private String createCookie(String webId){
		String cookie = "";
		cookie+="uuid=\"w:"+MD5Util.MD5Str(System.currentTimeMillis()+""+new Random().nextLong())+"\";"
				+"UM_distinctid=15e3c1f05ee666-0bbe3b28171d3a-3a3e5e06-1aeaa0-15e3c1f05efcf7; csrftoken="
				+MD5Util.MD5Str(System.currentTimeMillis()+""+new Random().nextLong());
		cookie+="; _ba=BA0.2-20170901-51d9e-u7pevbGksvtqRxBSkGdd; tt_track_id="+MD5Util.MD5Str(System.currentTimeMillis()+""+new Random().nextLong())+"; tt_webid="+webId+"; WEATHER_CITY=%E5%8C%97%E4%BA%AC; __tasessionId="+MD5Util.MD5Str(System.currentTimeMillis()+""+new Random().nextLong())+"; tt_webid="+webId+"; CNZZDATA1259612802=14562407-1504243000-null%7C1509682054";
		return cookie;
	}

	public void run() {
		CommonUtils commonUtils = new CommonUtils();
		List<ProxyIP> ips = new ArrayList<ProxyIP>();
		if("1".equals(ApplicationConfig.prop.getProperty("useProxy_taojin"))
				||"1".equals(ApplicationConfig.prop.getProperty("useProxy_toutiao"))){
			for (ProxyIP proxyIP : proxyIps) {
				ips = commonUtils.getHttpsProxy(proxyIP);
				if(ips!=null&&ips.size()>0){
					break;
				}
			}
			
		}
		this.proxyIps.addAll(ips);
		this.proxyIps_toutiao.addAll(ips);
		logger.info("【"+category+"】代理ip总数:" + proxyIps.size());
		Collections.shuffle(proxyIps);
		Collections.shuffle(proxyIps_toutiao);
		String apiCookie = ApplicationConfig.prop.getProperty("api_cookie");
		if(StringUtils.isEmpty(apiCookie)){
			apiCookie = "uuid=\"w:aee10691133d42f2868101b1935ced0a\"; __utma=24953151.1865537224.1479891391.1479974989.1479974989.1; __utmz=24953151.1479974989.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); csrftoken=6ca9eb799f6360f14cfe1edc22191550; tt_webid=36983044266; CNZZDATA1259612802=1771138202-1479887879-https%253A%252F%252Fwww.baidu.com%252F%7C1488508699; _ga=GA1.2.1865537224.1479891391; __tasessionId=x73ep83sz1488509821994";
		}
		
		String articleCookie = ApplicationConfig.prop.getProperty("article_cookie");
		if(StringUtils.isEmpty(articleCookie)){
			articleCookie = "uuid=\"w:4fead9ab1e85431d81079fdd682e4dcf\"; UM_distinctid=15e3c1f05ee666-0bbe3b28171d3a-3a3e5e06-1aeaa0-15e3c1f05efcf7; csrftoken=5160e729fb9229b2abe2bd577f1ddc57; _ba=BA0.2-20170901-51d9e-u7pevbGksvtqRxBSkGdd; tt_track_id=a7178a39f6084a7f388225663ae9af2a; tt_webid=6460691465168788573; WEATHER_CITY=%E5%8C%97%E4%BA%AC; __tasessionId=7769u1u8b1509685d50608; tt_webid=6460691465168788573; CNZZDATA1259612802=14562407-1504243000-null%7C1509682054";
		}
		// 设置请求头
		header.put("Host", "www.51taojinge.com");
		header.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0");
		header.put("Accept", "text/javascript, text/html, application/xml, text/xml, */*");
		header.put("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
		header.put("Accept-Encoding", "gzip, deflate");
		header.put("X-Requested-With", "XMLHttpRequest");
		header.put("Content-Type", "text/*, application/xml");
		header.put("Connection", "keep-alive");
		header.put("Cookie",apiCookie);

		header1.put("Host", "www.toutiao.com");
		header1.put("User-Agent",
				"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0");
		header1.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		header1.put("Accept-Language", "zh-CN,zh;q=0.8");
		header1.put("Accept-Encoding", "gzip, deflate, br");
		header1.put("Connection", "keep-alive");
		header1.put("Upgrade-Insecure-Requests", "1");
		header1.put("Cookie",articleCookie);
		// 获取本地的多个ip
		getLocalIps();
		long begin = new Date().getTime();
		sqlSessionFactory = MyBatisUtils.getSqlSessionFactory();
		sqlSession = sqlSessionFactory.openSession();
		articleMapper = sqlSession.getMapper(ArticleMapper.class);
		logger.info("----------访问栏目:" + category + "----------");
		otherCategory = category;
		if (otherCategory.equals("__all__")) {
			header.put("Referer", "http://www.toutiao.com/");
		} else {
			header.put("Referer", "http://www.51taojinge.com/index.php");
		}
		String testUrl = "http://www.51taojinge.com/wangyi/index.php"+("all".equals(category)?"":"?tag="+category+"&1=1");
		List<Map<String,String>> taojinList = new ArrayList<Map<String,String>>();
		// 是否使用代理ip
		if (!"0".equals(ApplicationConfig.prop.getProperty("useProxy_taojin"))) {
			// 设置代理ip
			taojinList = setProxyIp_taojin(testUrl);
			if (taojinList==null||taojinList.isEmpty()) {
				logger.info(category+":【淘金】没有可用ip，程序退出");
				return;
			}
		} else {
			// 设置本机的出口ip
			if(localIps!=null&&localIps.size()>0){
				taojinList = setOutIp_taojin(testUrl);
				if (taojinList == null || taojinList.isEmpty()) {
					logger.info(category+":【淘金】没有可用ip，程序退出");
					return;
				}
			}
		}
//		List<Map<String,String>> articleList = getArticleList("http://www.51taojinge.com/index.php?tag="+category+"&1=1");
		for (Map<String, String> map : taojinList) {
			String articleUrl = map.get("url");
			String tag = null;
			String catagory = StringUtils.isEmpty(map.get("category"))?"其他":map.get("category");
			String catagoryFlag = category;
			String content = null;
			String authorTitle = null;
			String contentText = "";
			String thumbnail = "";
			if (StringUtils.isNotEmpty(articleUrl)) {
				// 访问文章页面，获取其他信息
				Map<String, Object> articleInfo = null;
				if ((articleInfo=setProxyIp_toutiao(articleUrl))==null) {
					logger.info(category+":【网易】没有可用ip，程序退出");
				}
				if (articleInfo != null&&!articleInfo.containsKey("isArticle")) {
					tag = articleInfo.get("tag") == null ? "" : articleInfo.get("tag").toString();
					content = articleInfo.get("content") == null ? ""
							: articleInfo.get("content").toString();
					contentText = articleInfo.get("contentText") == null ? ""
							: articleInfo.get("contentText").toString();
					authorTitle = articleInfo.get("author") == null ? ""
							: articleInfo.get("author").toString();
					catagoryFlag = catagoryFlag.replace("news_", "");
					thumbnail = articleInfo.get("thumbnail")==null?"":articleInfo.get("thumbnail").toString();
					thumbnail = "null".equals(thumbnail)?"":thumbnail;
				} 
			}
			if (StringUtils.isNotEmpty(content) && StringUtils.isNotEmpty(contentText)
					&& contentText.length() >= 250) {
				// 查询文章是否已存在，如果已存在，则不插入，如果连续两篇已存在，则返回
				Map<String, Object> articleMap = new HashMap<String, Object>();
				articleMap.put("articleid", 1);
				String title = map.get("title");
				title = title.replace(" ", "@@@");
				title = commonUtils.simple2tw(title);
				title = title.replace("@@@"," ");
				articleMap.put("title", title);
				articleMap.put("catagory", commonUtils.simple2tw(catagory));
				articleMap.put("catagoryflag", !"all".equals(catagoryFlag)?catagoryFlag:(Constants.CATEGORY.get(catagory)==null?"other":Constants.CATEGORY.get(catagory)));
				String simple2tw = commonUtils.simple2tw(commonUtils.encodeHtml(content));
				content = commonUtils.decodeHtml(simple2tw);
				// 下载html中的图片
				String imgName = downloadThumbnail(thumbnail,new SimpleDateFormat("yyyyMMdd").format(new Date()));
				if(StringUtils.isNotEmpty(imgName)){
					if (!"0".equals(ApplicationConfig.prop.getProperty("replace_imgUrl_prefix"))) {
						String prefix = ApplicationConfig.prop.getProperty("imgUrl_prefix");
						if (StringUtils.isNotEmpty(prefix) && prefix.lastIndexOf("/") != prefix.length() - 1) {
							prefix = prefix + "/";
						}
						thumbnail = prefix + imgName;
					}
				}
				content = downloadPic(content,thumbnail);
				articleMap.put("content", content);
				articleMap.put("tags", commonUtils.simple2tw(tag));
				articleMap.put("author", commonUtils.simple2tw(authorTitle));
				articleMap.put("authorId", 1);
				articleMap.put("url", articleUrl);
				Date publishTime = new Date();
				try {
					publishTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(map.get("publishtime"));
				} catch (ParseException e) {
//					e.printStackTrace();
				}
				articleMap.put("publishtime", publishTime);
				articleMap.put("thumbnail", thumbnail);
				String day = new SimpleDateFormat("yyyyMMdd").format(publishTime);
				try {
					articleMap.put("date", new SimpleDateFormat("yyyyMMdd").parse(day).getTime()/1000);
				} catch (ParseException e1) {
				}
				try {
					articleMapper.insertArticle(articleMap);
					sqlSession.commit();
					totalCount++;
				} catch (Exception e) {
					if(e.getMessage()!=null&&e.getMessage().contains("Duplicate entry")){
						//重复抓取
						break;
					}
					existCount++;
				}
			}
		}
		sqlSession.close();
		long end = new Date().getTime();
		logger.info("抓取"+category+"结束,本次应更新"+taojinList.size()+",实际更新:" + totalCount + "条数据,用时:" + ((end - begin) / 1000 / 60) + "分");

	}

	public void getLocalIps() {
		String ips = ApplicationConfig.prop.getProperty("localIps");
		if (StringUtils.isNotEmpty(ips)) {
			ips = ips.replace("，", ",");
			ips = ips.trim();
			String[] split = ips.split(",");
			for (String ip : split) {
				localIps.add(ip);
			}
		}
	}

	/**
	 * 设置出口ip
	 */
	public List<Map<String, String>> setOutIp_taojin(String url) {
		proxyIP_taojin = null;
		for (String ip : localIps) {
			try {
				Runtime.getRuntime().exec(ApplicationConfig.prop.getProperty("ipshell") + " " + ip);
				logger.info("设置出口ip:" + ip);
				return checkIpCanUsed_taojin(url);
			} catch (IOException e) {
				logger.error(e.getMessage());
			}
		}
		return null;
	}
	
	public Map<String, Object> setOutIp_toutiao(String url) {
		proxyIP_toutiao = null;
		for (String ip : localIps) {
			try {
				Runtime.getRuntime().exec(ApplicationConfig.prop.getProperty("ipshell") + " " + ip);
				logger.info("设置出口ip:" + ip);
				Map<String, Object> articleInfo = null;
				if((articleInfo = checkIpCanUsed_toutiao(url))!=null){
					return articleInfo;
				}
			} catch (IOException e) {
				logger.error(e.getMessage());
			}
		}
		return null;
	}


	public String downloadPic(String html,String thumbnail) {
		String prefix = ApplicationConfig.prop.getProperty("imgUrl_prefix");
		if (StringUtils.isNotEmpty(prefix) && prefix.lastIndexOf("/") != prefix.length() - 1) {
			prefix = prefix + "/";
		}
		Document document = Jsoup.parse(html);
		Elements imgs = document.getElementsByTag("img");
		if (imgs != null && imgs.size() > 0) {
			for (Element img : imgs) {
				String imgUrl = img.attr("src");
				String middle = "";
				if(imgUrl.contains("//")){
					middle = imgUrl.substring(imgUrl.lastIndexOf("//")+2, imgUrl.lastIndexOf("/")+1);
				}else{
					middle = imgUrl.substring(0, imgUrl.lastIndexOf("/")+1);
				}
				if(middle.indexOf("/")!=middle.lastIndexOf("/")){
					middle = middle.substring(middle.indexOf("/")+1);
				}else{
					middle = "";
				}
				String imgName = download(imgUrl);
				if (StringUtils.isNotEmpty(imgName)) {
					if(thumbnail!=null&&thumbnail.indexOf(imgName)!=-1){
						img.attr("src", thumbnail);
					}else{
						img.attr("src", prefix + middle + imgName);
					}
				}
			}
		}
		if ("0".equals(ApplicationConfig.prop.getProperty("replace_imgUrl_prefix"))) {
			// 不修改url前缀
			return html;
		} else {
			return document.getElementsByTag("body").html();
		}
	}

	public String download(String imgUrl) {
		try {
			String imgName = imgUrl.substring(imgUrl.lastIndexOf("/") + 1);
			if ("1".equals(ApplicationConfig.prop.getProperty("download_img"))) {
				// 下载图片
				URL url = new URL(imgUrl);
				HttpURLConnection conn = (HttpURLConnection) url.openConnection();
				conn.setConnectTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
				conn.setReadTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
				DataInputStream dataInputStream = new DataInputStream(conn.getInputStream());
				if(!imgName.toLowerCase().endsWith(".jpg")
						&&!imgName.toLowerCase().endsWith(".jpeg")
						&&!imgName.toLowerCase().endsWith(".png")
						&&!imgName.toLowerCase().endsWith(".gif")){
					imgName = imgName + ".jpg";
				}
				File dir = new File(ApplicationConfig.prop.getProperty("img_dir"));
				if (!dir.exists()) {
					dir.mkdirs();
				}
				File file = new File(ApplicationConfig.prop.getProperty("img_dir") + File.separator + imgName);
				if (!file.exists()) {
					file.createNewFile();
				}
				FileOutputStream fileOutputStream = new FileOutputStream(file);
				byte[] buffer = new byte[1024];
				int length;
				while ((length = dataInputStream.read(buffer)) > 0) {
					fileOutputStream.write(buffer, 0, length);
				}
				dataInputStream.close();
				fileOutputStream.close();
			}
			return imgName;
		} catch (MalformedURLException e) {
			logger.error("error:" + e.getMessage());
		} catch (IOException e) {
			logger.error("error:" + e.getMessage());
		}
		return null;
	}
	
	public String downloadThumbnail(String imgUrl,String date) {
		if(StringUtils.isEmpty(imgUrl)){
			return null;
		}
		String imgName = imgUrl.substring(imgUrl.lastIndexOf("/") + 1);
		if(!imgName.toLowerCase().endsWith(".jpg")
				&&!imgName.toLowerCase().endsWith(".jpeg")
				&&!imgName.toLowerCase().endsWith(".png")
				&&!imgName.toLowerCase().endsWith(".gif")){
			imgName = imgName + ".jpg";
		}
		try {
			// 下载图片
			URL url = new URL(imgUrl);
			HttpURLConnection conn = (HttpURLConnection) url.openConnection();
			conn.setConnectTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
			conn.setReadTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
			DataInputStream dataInputStream = new DataInputStream(conn.getInputStream());
			File dir = new File(ApplicationConfig.prop.getProperty("img_dir"));
			if (!dir.exists()) {
				dir.mkdirs();
			}
			dir = new File(dir.getAbsolutePath()+File.separator+date);
			if (!dir.exists()) {
				dir.mkdirs();
			}
			File file = new File(dir.getAbsolutePath() + File.separator + imgName);
			if (!file.exists()) {
				file.createNewFile();
			}
			FileOutputStream fileOutputStream = new FileOutputStream(file);
			byte[] buffer = new byte[1024];
			int length;
			while ((length = dataInputStream.read(buffer)) > 0) {
				fileOutputStream.write(buffer, 0, length);
			}
			dataInputStream.close();
			fileOutputStream.close();
		} catch (Exception e) {
//			logger.error("url:"+imgUrl+" error:" + e.getMessage());
		}
		return date+"/"+imgName;
	}
	
	public Map<String, Object> getArticleInfo(String url) {
		CommonUtils commonUtils = new CommonUtils();
		if (!url.contains("dy.163.com")) {
			url = "http://www.dy.163.com" + url;
		}
		header1.put("Cookie", createCookie(matchNum(url)));
		try {
			Map<String, Object> map = new HashMap<String, Object>();
			Connection connect = Jsoup.connect(url).headers(header1).timeout(3000).followRedirects(true);
			if(proxyIP_toutiao!=null&&!"127.0.0.1".equals(proxyIP_toutiao.getIp())){
				connect = connect.proxy(proxyIP_toutiao.getIp(), proxyIP_toutiao.getPort());
			}
			Document document;
			document = connect.ignoreContentType(true).get();
			Element contentEle = document.getElementById("content");
			String tag = "";
			String content = contentEle.html();
			String contentText = "";
			String thumbnail = "";
			if (StringUtils.isNotEmpty(content)) {
				contentText = contentEle.text();
				Elements imgs = contentEle.getElementsByTag("img");
				if(imgs!=null&&imgs.size()>0){
					String src = "";
					for (Element element : imgs) {
						String imgSrc = "";
						if(StringUtils.isEmpty(src)){
							src = element.attr("src");
						}
						imgSrc = element.attr("src");
						if(StringUtils.isEmpty(imgSrc)){
							imgSrc = element.attr("data-src");
							if(StringUtils.isNotEmpty(imgSrc)){
								element.attr("src", imgSrc);
								element.attr("data-src","");
								if(StringUtils.isEmpty(src)){
									src = imgSrc;
								}
							}
						}
					}
					if(!src.startsWith("http")){
						if(!src.startsWith("//")){
							src = "https://"+src;
						}else{
							src = "https:"+src;
						}
					}
					thumbnail = src;
				}
			}
			map.put("author", "");
			map.put("description", "");
			map.put("content", content);
			map.put("contentText", contentText);
			map.put("section", "");
			map.put("tag", tag);
			map.put("thumbnail", thumbnail);
			return map;
		} catch (Exception e) {
//			logger.error("访问文章页失败:" + url + "  原因" + e.getMessage());
		}
		return null;
	}

	
	private String matchNum(String url) {
		Pattern pattern = Pattern.compile("[0-9]{19}");
		Matcher matcher = pattern.matcher(url);
		if(matcher.find()){
			return matcher.group(0);
		}
		return "";
	}
	
	public Map<String, Object> getArticleInfoWithErrorLog(String url) {
		CommonUtils commonUtils = new CommonUtils();
		if (!url.contains("toutiao.com")) {
			url = "http://www.toutiao.com" + url;
		}
		header1.put("Cookie", createCookie(matchNum(url)));
		try {
			Map<String, Object> map = new HashMap<String, Object>();
			Connection connect = Jsoup.connect(url).headers(header1).followRedirects(true);
			if(proxyIP_toutiao!=null&&!"127.0.0.1".equals(proxyIP_toutiao.getIp())){
				connect = connect.proxy(proxyIP_toutiao.getIp(), proxyIP_toutiao.getPort());
			}
			Document document;
			document = connect.ignoreContentType(true).get();
			Elements scripts = document.getElementsByTag("script");
			String tag = "";
			String content = "";
			if(scripts!=null&&scripts.size()>0){
				int tagIndex = -1;
				int articleIndex = -1;
				for (int i=0;i<scripts.size()-1;i++) {
					if(scripts.get(i).html().contains("var __ptags")){
						tagIndex = i;
					}
					if(scripts.get(i).html().contains("var BASE_DATA")){
						articleIndex = i;
					}
				}
				String tagJs = "";
				if(tagIndex>0){
					tagJs = scripts.get(tagIndex).html();
					tagJs = "function getTag(){var __ptags=[]; "+tagJs+" var str=''; for(var k in __ptags){str=str+__ptags[k]['name']+','}return str;}";
					try {
						tag = commonUtils.runJSAndReturn(tagJs,"getTag");
					} catch (Exception e) {
						logger.error("js error:"+url+" "+e.getMessage());
					}
					if(StringUtils.isNotEmpty(tag)){
						tag = tag.substring(0,tag.lastIndexOf(","));
					}
				}
				String articleJs = "";
				if(articleIndex>0){
					articleJs = scripts.get(articleIndex).html();
					if(articleJs.lastIndexOf("articleInfo:")!=-1){
						articleJs = articleJs.substring(articleJs.lastIndexOf("articleInfo:"));
						articleJs = articleJs.substring(0, articleJs.indexOf("commentInfo:"));
					}
					tagJs = "function getContent(){var __ptags=[];var BASE_DATA={"+articleJs+"commentInfo:''}; var result = ''; if(BASE_DATA.articleInfo.tagInfo.tags){for(var k in BASE_DATA.articleInfo.tagInfo.tags){result=result+BASE_DATA.articleInfo.tagInfo.tags[k]['name']+','} } return result;}";
					articleJs = "function getContent(){var __ptags=[];var BASE_DATA={"+articleJs+"commentInfo:''}; return BASE_DATA.articleInfo.content;}";
					try {
						content = commonUtils.runJSAndReturn(articleJs, "getContent");
					} catch (Exception e) {
						logger.error("js error:"+url+" "+e.getMessage());
					}
					if(StringUtils.isEmpty(tag)){
						try {
							tag = commonUtils.runJSAndReturn(tagJs, "getContent");
						} catch (Exception e) {
							logger.error("js error:"+url+" "+e.getMessage());
						}
						if(StringUtils.isNotEmpty(tag)){
							tag = tag.substring(0,tag.lastIndexOf(","));
						}
					}
					if(StringUtils.isNotEmpty(content)){
						content = commonUtils.decodeHtml(content);
					}
				}
			}
			String contentText = "";
			String thumbnail = "";
			if (StringUtils.isNotEmpty(content)) {
				Document contentEle = Jsoup.parse(content);
				contentText = contentEle.text();
				Elements imgs = contentEle.getElementsByTag("img");
				if(imgs!=null&&imgs.size()>0){
					String src = imgs.get(0).attr("src");
					if(!src.startsWith("http")){
						if(!src.startsWith("//")){
							src = "http://"+src;
						}else{
							src = "http:"+src;
						}
					}
					thumbnail = src;
				}
			}
			map.put("author", "");
			map.put("description", "");
			map.put("content", content);
			map.put("contentText", contentText);
			map.put("section", "");
			map.put("tag", tag);
			map.put("thumbnail", thumbnail);
			return map;
		} catch (Exception e) {
			logger.error("访问文章页失败:" + url + "  原因" + e.getMessage());
		}
		return null;
	}

	public List<Map<String, String>> setProxyIp_taojin(String url) {
		// 取1个页面访问，如果都访问不了的话，则换ip，如果都能访问，则设置为代理ip，如果都没有的话则使用本机ip
		if (proxyIps != null && proxyIps.size() > 0) {
			Iterator<ProxyIP> it = proxyIps.iterator();
			int index = 0;
			while (it.hasNext()) {
				index++;
				ProxyIP ip = it.next();
				proxyIP_taojin = ip;
				List<Map<String, String>> list = checkIpCanUsed_taojin(url);
				if (list!=null&&!list.isEmpty()) {
//					logger.info(category+":【淘金】第"+index+"个代理IP：" + ip.getIp() + "可用");
					return list;
				} else {
					it.remove();
					proxyIP_taojin = null;
				}
			}
		}
		// 先判断本机ip能否访问
		return setOutIp_taojin(url);
	}
	public Map<String, Object> setProxyIp_toutiao(String url) {
		Map<String, Object> articleInfo = null;
		// 取1个页面访问，如果都访问不了的话，则换ip，如果都能访问，则设置为代理ip，如果都没有的话则使用本机ip
		if (proxyIps_toutiao != null && proxyIps_toutiao.size() > 0) {
			Iterator<ProxyIP> it = proxyIps_toutiao.iterator();
			int index = 0;
			while (it.hasNext()) {
				index++;
				ProxyIP ip = it.next();
				proxyIP_toutiao = ip;
				if ((articleInfo=checkIpCanUsed_toutiao(url))!=null) {
//					logger.info(category+":【头条】第"+index+"个代理IP：" + ip.getIp() + "可用");
					return articleInfo;
				} else {
					it.remove();
					proxyIP_toutiao = null;
				}
			}
		}
		// 先判断本机ip能否访问
		if ((articleInfo=setOutIp_toutiao(url))!=null) {
			return articleInfo;
		}
		return null;
	}
	
	private List<Map<String, String>> getArticleList(String url){
		Connection con = Jsoup.connect(url).headers(header).followRedirects(true).ignoreContentType(true);
		List<Map<String, String>> list = new ArrayList<Map<String,String>>();
		if(proxyIP_taojin!=null&&!"127.0.0.1".equals(proxyIP_taojin.getIp())){
			con = con.proxy(proxyIP_taojin.getIp(), proxyIP_taojin.getPort());
		}
		Document doc = null;
		try {
			doc = con.get();
		} catch (IOException e) {
			return list;
		}
		Elements tbodys = doc.getElementsByTag("tbody");
		if(tbodys!=null&&tbodys.size()>0){
			Elements trs = tbodys.get(0).getElementsByTag("tr");
			if(trs!=null&&trs.size()>0){
				for (Element tr : trs) {
					Elements td = tr.getElementsByTag("td");
					if(td!=null&&td.size()>=7){
						Map<String, String> map = new HashMap<String, String>();
						map.put("title", td.get(1).text());
						map.put("url", td.get(1).getElementsByTag("a").get(0).attr("href"));
						map.put("category",td.get(4).text());
						String time = td.get(5).text();
						map.put("publishtime", time);
						list.add(map);
					}
				}
			}
		}
		return list;
	}

	public List<Map<String, String>> checkIpCanUsed_taojin(String url) {
		List<Map<String,String>> list = getArticleList(url);
		if(list!=null&&!list.isEmpty()){
			return list;
		}
		return null;
	}
	
	public Map<String, Object> checkIpCanUsed_toutiao(String url) {
		Map<String, Object> articleInfo = getArticleInfo(url);
		if(articleInfo!=null&&StringUtils.isNotEmpty(articleInfo.get("content")==null?"":articleInfo.get("content").toString())){
			return articleInfo;
		}
		return null;
	}
	
}
